{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import tensorflow.keras.layers as layer\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "# from tensorflow.keras.models import Model\n",
    "from tensorflow.keras import Model\n",
    "import tensorflow.keras.backend as K\n",
    "from sklearn.model_selection import StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>I1</th>\n",
       "      <th>I2</th>\n",
       "      <th>I3</th>\n",
       "      <th>I4</th>\n",
       "      <th>I5</th>\n",
       "      <th>I6</th>\n",
       "      <th>I7</th>\n",
       "      <th>I8</th>\n",
       "      <th>I9</th>\n",
       "      <th>...</th>\n",
       "      <th>C17</th>\n",
       "      <th>C18</th>\n",
       "      <th>C19</th>\n",
       "      <th>C20</th>\n",
       "      <th>C21</th>\n",
       "      <th>C22</th>\n",
       "      <th>C23</th>\n",
       "      <th>C24</th>\n",
       "      <th>C25</th>\n",
       "      <th>C26</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1382.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>181.0</td>\n",
       "      <td>...</td>\n",
       "      <td>e5ba7672</td>\n",
       "      <td>f54016b9</td>\n",
       "      <td>21ddcdc9</td>\n",
       "      <td>b1252a9d</td>\n",
       "      <td>07b5194c</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3a171ecb</td>\n",
       "      <td>c5c50484</td>\n",
       "      <td>e8b83407</td>\n",
       "      <td>9727dd16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>102.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>07c540c4</td>\n",
       "      <td>b04e4670</td>\n",
       "      <td>21ddcdc9</td>\n",
       "      <td>5840adea</td>\n",
       "      <td>60f6221e</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3a171ecb</td>\n",
       "      <td>43f13e8b</td>\n",
       "      <td>e8b83407</td>\n",
       "      <td>731c3655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>767.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>245.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8efede7f</td>\n",
       "      <td>3412118d</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>e587c466</td>\n",
       "      <td>ad3062eb</td>\n",
       "      <td>3a171ecb</td>\n",
       "      <td>3b183c5c</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>893</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4392.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1e88c74f</td>\n",
       "      <td>74ef3502</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6b3a5ca6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3a171ecb</td>\n",
       "      <td>9117a34a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1e88c74f</td>\n",
       "      <td>26b3c7a7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21c9516a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32c7478e</td>\n",
       "      <td>b34f3128</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   label   I1   I2    I3    I4      I5    I6    I7   I8     I9  ...       C17  \\\n",
       "0      0  1.0    1   5.0   0.0  1382.0   4.0  15.0  2.0  181.0  ...  e5ba7672   \n",
       "1      0  2.0    0  44.0   1.0   102.0   8.0   2.0  2.0    4.0  ...  07c540c4   \n",
       "2      0  2.0    0   1.0  14.0   767.0  89.0   4.0  2.0  245.0  ...  8efede7f   \n",
       "3      0  NaN  893   NaN   NaN  4392.0   NaN   0.0  0.0    0.0  ...  1e88c74f   \n",
       "4      0  3.0   -1   NaN   0.0     2.0   0.0   3.0  0.0    0.0  ...  1e88c74f   \n",
       "\n",
       "        C18       C19       C20       C21       C22       C23       C24  \\\n",
       "0  f54016b9  21ddcdc9  b1252a9d  07b5194c       NaN  3a171ecb  c5c50484   \n",
       "1  b04e4670  21ddcdc9  5840adea  60f6221e       NaN  3a171ecb  43f13e8b   \n",
       "2  3412118d       NaN       NaN  e587c466  ad3062eb  3a171ecb  3b183c5c   \n",
       "3  74ef3502       NaN       NaN  6b3a5ca6       NaN  3a171ecb  9117a34a   \n",
       "4  26b3c7a7       NaN       NaN  21c9516a       NaN  32c7478e  b34f3128   \n",
       "\n",
       "        C25       C26  \n",
       "0  e8b83407  9727dd16  \n",
       "1  e8b83407  731c3655  \n",
       "2       NaN       NaN  \n",
       "3       NaN       NaN  \n",
       "4       NaN       NaN  \n",
       "\n",
       "[5 rows x 40 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('./data/criteo_sampled_data.csv')\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = train.columns[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dense_feats = [f for f in cols if f[0] == 'I']\n",
    "sparse_feats = [f for f in cols if f[0] == 'C']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_dense_feats(data, feats):\n",
    "    d = data.copy()\n",
    "    d = d[feats].fillna(0)\n",
    "    for f in feats:\n",
    "        d[f] = d[f].apply(lambda x: np.log(x+1) if x>-1 else -1)\n",
    "    return d\n",
    "data_dense = process_dense_feats(train, dense_feats)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_sparse_feats(data, feats):\n",
    "    d = data.copy()\n",
    "    d = d[feats].fillna('-1')\n",
    "    for f in feats:\n",
    "        d[f] = LabelEncoder().fit_transform(d[f])\n",
    "    return d\n",
    "data_sparse = process_sparse_feats(train, sparse_feats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_data = pd.concat([data_dense, data_sparse], axis=1)\n",
    "total_data['label'] = train['label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 如果你只是想对流经该层的数据做个变换，而这个变换本身没有什么需要学习的参数，那么直接用Lambda Layer是最合适的了"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 获取类别型特征的大小\n",
    "sparse_feat_config= dict()\n",
    "for col in sparse_feats:\n",
    "    sparse_feat_config[col] = total_data[col].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 获取类别型特征的大小\n",
    "dense_feat_config= []\n",
    "for col in dense_feats:\n",
    "    dense_feat_config.append(col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构造验证集和训练集\n",
    "train_data = total_data.loc[:500000-1]\n",
    "valid_data = total_data.loc[500000:]\n",
    "\n",
    "train_dense_x = [train_data[f].values for f in dense_feats]#  train_data[dense_feats] \n",
    "train_sparse_x = [train_data[f].values for f in sparse_feats] # train_data[sparse_feats] # \n",
    "train_label = train_data['label'].values\n",
    "train_label = tf.cast(train_label, tf.int32)\n",
    "\n",
    "val_dense_x = [valid_data[f].values for f in dense_feats] # valid_data[dense_feats]   \n",
    "val_sparse_x = [valid_data[f].values for f in sparse_feats] # valid_data[sparse_feats]\n",
    "val_label = valid_data['label'].values\n",
    "val_label = tf.cast(val_label, tf.int32)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构造训练集和测试集\n",
    "def make_data(total_data,idx):\n",
    "    train_data = total_data.loc[idx,:]\n",
    "    train_dense_x = [train_data[f].values for f in dense_feats]\n",
    "    train_sparse_x = [train_data[f].values for f in sparse_feats]\n",
    "    train_label = train_data['label'].values\n",
    "    return train_sparse_x,train_dense_x,train_label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 写法一\n",
    "继承layer,定义不同功能的层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 独立层：\n",
    "# sparse 嵌入层\n",
    "class Embedding_sparse_layer(tf.keras.layers.Layer):\n",
    "    def __init__(self,sparse_feat_config, embeding_shape):\n",
    "        super(Embedding_sparse_layer, self).__init__()\n",
    "        # l2正则化\n",
    "        self.reg_1 = tf.keras.regularizers.l2(0.1)\n",
    "        self.embed_first = {}\n",
    "        self.sparse_feat_config = sparse_feat_config\n",
    "        self.embeding_shape = embeding_shape\n",
    "        self.sparse_feat = list(sparse_feat_config.keys())\n",
    "        for key, value in self.sparse_feat_config.items():\n",
    "            self.embed_first[key] = layer.Embedding(value+1,self.embeding_shape, \n",
    "                                                    embeddings_regularizer=self.reg_1, \n",
    "                                                    name='embed'+key)\n",
    "    def call(self,x_sparse):\n",
    "        embed_lookup_first = []\n",
    "        for i,key in enumerate(self.sparse_feat):\n",
    "\n",
    "            _embed = self.embed_first[key](x_sparse[i])\n",
    "\n",
    "            embed_lookup_first.append(_embed)\n",
    "\n",
    "        return embed_lookup_first\n",
    "    \n",
    "    def get_config(self):\n",
    "\n",
    "        config = super().get_config().copy()\n",
    "        config.update({\n",
    "            'sparse_feat_config': self.sparse_feat_config,\n",
    "            'embeding_shape': self.embeding_shape,\n",
    "        })\n",
    "        return config\n",
    "# t = Embedding_dense(sparse_feat_config,1)\n",
    "# y = t(inputs_sparse)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 独立层：\n",
    "# dense 嵌入层\n",
    "class Embedding_dense_layer(tf.keras.layers.Layer):\n",
    "    def __init__(self, embeding_shape):\n",
    "        super(Embedding_dense_layer, self).__init__()\n",
    "        self.embeding_shape = embeding_shape\n",
    "        self.embeds = []\n",
    " \n",
    "    def build(self,input_shape):\n",
    "#         print('input',input_shape)\n",
    "        for _ in enumerate(input_shape):\n",
    "            embed = tf.Variable(\n",
    "                        initial_value = tf.random_normal_initializer()(shape=(1,self.embeding_shape),dtype='float32'),\n",
    "                        trainable=True\n",
    "                     )\n",
    "#             embed = tf.Variable(lambda:tf.random.truncated_normal(shape=(1,self.embeding_shape), stddev=0.01))\n",
    "            self.embeds.append(embed)\n",
    "            \n",
    "    def call(self,x_dense):\n",
    "        dense_kd_embed = []\n",
    "        for i,_input in enumerate(x_dense):\n",
    "#             f = _input.name[:4]\n",
    "#             embed = tf.Variable(lambda:tf.random.truncated_normal(shape=(1,self.embeding_shape), stddev=0.01,name='dense'+str(i) ))\n",
    "            \n",
    "            scaled_embed = tf.expand_dims(_input*self.embeds[i], axis=1)\n",
    "            dense_kd_embed.append(scaled_embed)\n",
    "        return dense_kd_embed\n",
    "    \n",
    "    def get_config(self):\n",
    "\n",
    "        config = super().get_config().copy()\n",
    "        config.update({\n",
    "#             'sparse_feat_config': self.sparse_feat_config,\n",
    "            'embeding_shape': self.embeding_shape,\n",
    "        })\n",
    "        return config\n",
    "# t = Embedding_dense(sparse_feat_config,1)\n",
    "# y = t(inputs_sparse)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# attention 层\n",
    "class attention_layer(tf.keras.layers.Layer):\n",
    "    def __init__(self,d,n_attention_head):\n",
    "        super(attention_layer,self).__init__()\n",
    "        self.n_attention_head = n_attention_head\n",
    "#         self.embed_map = embed_map\n",
    "        self.dense_q = []\n",
    "        self.dense_k = []\n",
    "        self.dense_v = []  \n",
    "        self.attention_layers = []\n",
    "        for i in range(self.n_attention_head):\n",
    "#             print('init',i)\n",
    "            # (self.embed_map)\n",
    "            self.dense_q.append(layer.Dense(d))\n",
    "            self.dense_k.append(layer.Dense(d))\n",
    "            self.dense_v.append(layer.Dense(d))\n",
    "            self.attention_layers.append(layer.Attention())    \n",
    "    def call(self, embed_map):\n",
    "       # embed_map shape (None, 39, 8),\n",
    "        attention_heads = []\n",
    "#         print('ggggg',n)\n",
    "        for i in range(self.n_attention_head):\n",
    "#             print('i',i)\n",
    "            attention_output = self.attention_layers[i]([self.dense_q[i](embed_map), self.dense_k[i](embed_map), self.dense_k[i](embed_map)])\n",
    "            attention_heads.append(attention_output)\n",
    "        if self.n_attention_head > 1:\n",
    "            muti_attention_output = layer.Concatenate(axis=-1)(attention_heads)\n",
    "        else:\n",
    "            muti_attention_output = attention_output\n",
    "        return muti_attention_output\n",
    "    def get_config(self):\n",
    "        config = super().get_config().copy()\n",
    "        config.update({\n",
    "            'n_attention_head': self.n_attention_head,\n",
    "            'd':self.d\n",
    "        })\n",
    "        return config  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AutoInt:\n",
    "    def __init__(self,sparse_feat_config,dense_feats):\n",
    "        \n",
    "        self.sparse_feat_config= sparse_feat_config\n",
    "        self.inputs_sparse, self.inputs_dense = self.build_input(sparse_feat_config,dense_feats)\n",
    "        \n",
    "#         self.firsr_cross_dense = firsr_cross_dense(self.sparse_feat_config)\n",
    "        self.Embedding_sparse_layer = Embedding_sparse_layer(self.sparse_feat_config,8)\n",
    "        self.Embedding_dense_layer = Embedding_dense_layer(8)\n",
    "        # 通过循环建立多个交互层\n",
    "        self.attention_layers = []\n",
    "        for i in range(2):\n",
    "            self.attention_layers.append(attention_layer(6,3))  # 第一个参数表示映射维度，第二个表示头的个数\n",
    "        \n",
    "#         self.DNN = DNN([128,128,64])\n",
    "        \n",
    "        self.AutoInt =  self.build_model() \n",
    "    def build_input(self,sparse_feat_config,dense_feats):\n",
    "        inputs_sparse = []\n",
    "        inputs_dense = []\n",
    "        for key in sparse_feat_config:\n",
    "            inputs_sparse.append(layer.Input(shape=(1,),name=key))\n",
    "        for key in dense_feats:\n",
    "            inputs_dense.append(layer.Input(shape=(1,),name=key))\n",
    "        \n",
    "        return inputs_sparse, inputs_dense\n",
    "\n",
    "    def build_model(self, num_lays = 3):\n",
    "        # DCN输入部分\n",
    "        # sparse特征嵌入\n",
    "        sparse_embed_lookup =  self.Embedding_sparse_layer(self.inputs_sparse) \n",
    "        dense_embed_lookup =  self.Embedding_dense_layer(self.inputs_dense) \n",
    "        input_embeds = sparse_embed_lookup + dense_embed_lookup\n",
    "        embed_map = layer.Concatenate(axis=1)(input_embeds) \n",
    "        \n",
    "        x_l = embed_map\n",
    "        for attention_layer in self.attention_layers:\n",
    "              x_l = attention_layer(x_l)\n",
    "        autoint_layer = layer.Flatten()(x_l)\n",
    "        output_layer = layer.Dense(1, activation=\"sigmoid\")(autoint_layer)\n",
    "#         return output_layer\n",
    "        # 初始化模型\n",
    "        model = Model(self.inputs_sparse + self.inputs_dense, outputs=output_layer)\n",
    "        model.compile(optimizer = tf.keras.optimizers.RMSprop(learning_rate=1e-3),\n",
    "                      loss= 'binary_crossentropy',\n",
    "                      metrics=['AUC'])\n",
    "        return model\n",
    "    \n",
    "\n",
    "    def train(self,train_data,train_label,valid_data, valid_label,batch_size,epochs,callbacks):\n",
    "        self.AutoInt.fit(train_data,train_label,\n",
    "                  batch_size=batch_size, epochs=epochs, verbose=1, \n",
    "                  validation_data=(valid_data, valid_label),\n",
    "                  callbacks = callbacks\n",
    "                 )        \n",
    "        \n",
    "# AutoInt(sparse_feat_config, dense_feats).build_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold: 0\n",
      "input [TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1])]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/indexed_slices.py:434: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "38/38 [==============================] - ETA: 0s - loss: 146.9688 - auc: 0.5422\n",
      "Epoch 00001: val_loss improved from inf to 11.01196, saving model to ./model/0.h5\n",
      "38/38 [==============================] - 31s 807ms/step - loss: 146.9688 - auc: 0.5422 - val_loss: 11.0120 - val_auc: 0.6388 - lr: 0.0010\n",
      "fold: 1\n",
      "input [TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1])]\n",
      "38/38 [==============================] - ETA: 0s - loss: 146.8706 - auc: 0.5253\n",
      "Epoch 00001: val_loss improved from inf to 11.00314, saving model to ./model/1.h5\n",
      "38/38 [==============================] - 30s 796ms/step - loss: 146.8706 - auc: 0.5253 - val_loss: 11.0031 - val_auc: 0.5952 - lr: 0.0010\n",
      "fold: 2\n",
      "input [TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1]), TensorShape([None, 1])]\n",
      "11/38 [=======>......................] - ETA: 18s - loss: 333.4923 - auc: 0.5035"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-20-0ec77026a0fe>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     24\u001b[0m     AutoInt_ctr.train(train_sparse_x+train_dense_x,train_label,\n\u001b[1;32m     25\u001b[0m                  \u001b[0mval_sparse_x\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mval_dense_x\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mval_label\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m                 12800,1, callbacks=callbacks)    \n\u001b[0m",
      "\u001b[0;32m<ipython-input-18-5662c738423a>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, train_data, train_label, valid_data, valid_label, batch_size, epochs, callbacks)\u001b[0m\n\u001b[1;32m     52\u001b[0m                   \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m                   \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalid_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_label\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 54\u001b[0;31m                   \u001b[0mcallbacks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     55\u001b[0m                  )        \n\u001b[1;32m     56\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36m_method_wrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     64\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_method_wrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_in_multi_worker_mode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m     \u001b[0;31m# Running inside `run_distribute_coordinator` already.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m    846\u001b[0m                 batch_size=batch_size):\n\u001b[1;32m    847\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 848\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    849\u001b[0m               \u001b[0;31m# Catch OutOfRangeError for Datasets of unknown size.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    850\u001b[0m               \u001b[0;31m# This blocks until the batch has finished executing.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    578\u001b[0m         \u001b[0mxla_context\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mExit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    579\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 580\u001b[0;31m       \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    581\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    582\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mtracing_count\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    609\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    610\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 611\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    612\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    613\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2418\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2419\u001b[0m       \u001b[0mgraph_function\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2420\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2421\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2422\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[0;34m(self, args, kwargs)\u001b[0m\n\u001b[1;32m   1663\u001b[0m          if isinstance(t, (ops.Tensor,\n\u001b[1;32m   1664\u001b[0m                            resource_variable_ops.BaseResourceVariable))),\n\u001b[0;32m-> 1665\u001b[0;31m         self.captured_inputs)\n\u001b[0m\u001b[1;32m   1666\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1667\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_flat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1744\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1745\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1746\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1747\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1748\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    596\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    597\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 598\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    599\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    600\u001b[0m           outputs = execute.execute_with_cancellation(\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 60\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     61\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# 五折交叉 + 提前停止 + 保存模型\n",
    "# tf.compat.v1.disable_eager_execution() 加入会报错\n",
    "\n",
    "skf = StratifiedKFold(n_splits = 5, random_state=1996, shuffle = True)\n",
    "for idx, (train_idx, val_idx) in enumerate(skf.split(total_data,total_data['label'])):\n",
    "    print('fold:',idx)\n",
    "    K.clear_session()\n",
    "    train_sparse_x,train_dense_x,train_label = make_data(total_data,train_idx)\n",
    "    val_sparse_x,val_dense_x,val_label = make_data(total_data,val_idx) \n",
    "    # 定义回调\n",
    "    \n",
    "    # 保存模型\n",
    "    file_path = f'./model/{idx}.h5'\n",
    "\n",
    "    checkpoint = tf.keras.callbacks.ModelCheckpoint(file_path, monitor='val_loss', verbose=1, save_best_only=True,save_weights_only=True, mode='min')\n",
    "    # metric 不提高时，减小学习率\n",
    "    reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.8, patience=1, min_lr=0.0001, verbose=1)\n",
    "    # val_loss 连续两次提升小于 1e-2，提前停止\n",
    "    earlystopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=2,verbose=1, mode='auto')\n",
    "    callbacks = [checkpoint, reduce_lr, earlystopping]\n",
    "\n",
    "    # 初始化模型\n",
    "    AutoInt_ctr = AutoInt(sparse_feat_config, dense_feats)\n",
    "    AutoInt_ctr.train(train_sparse_x+train_dense_x,train_label,\n",
    "                 val_sparse_x+val_dense_x,val_label,\n",
    "                12800,1, callbacks=callbacks)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.2748378 ],\n",
       "       [0.2906196 ],\n",
       "       [0.27955478],\n",
       "       ...,\n",
       "       [0.28339756],\n",
       "       [0.26656428],\n",
       "       [0.25270134]], dtype=float32)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 模型的加载与预测\n",
    "AutoInt_ctr = AutoInt(sparse_feat_config, dense_feats).build_model()\n",
    "AutoInt_ctr.load_weights('./model/0.h5')\n",
    "AutoInt_ctr.predict(val_sparse_x+val_dense_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_4\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "C1 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C2 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C3 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C4 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C5 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C6 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C7 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C8 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C9 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C10 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C11 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C12 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C13 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C14 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C15 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C16 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C17 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C18 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C19 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C20 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C21 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C22 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C23 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C24 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C25 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C26 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I1 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I2 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I3 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I4 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I5 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I6 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I7 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I8 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I9 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I10 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I11 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I12 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I13 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_sparse_layer_2 (Embed [(None, 1, 8), (None 7085680     C1[0][0]                         \n",
      "                                                                 C2[0][0]                         \n",
      "                                                                 C3[0][0]                         \n",
      "                                                                 C4[0][0]                         \n",
      "                                                                 C5[0][0]                         \n",
      "                                                                 C6[0][0]                         \n",
      "                                                                 C7[0][0]                         \n",
      "                                                                 C8[0][0]                         \n",
      "                                                                 C9[0][0]                         \n",
      "                                                                 C10[0][0]                        \n",
      "                                                                 C11[0][0]                        \n",
      "                                                                 C12[0][0]                        \n",
      "                                                                 C13[0][0]                        \n",
      "                                                                 C14[0][0]                        \n",
      "                                                                 C15[0][0]                        \n",
      "                                                                 C16[0][0]                        \n",
      "                                                                 C17[0][0]                        \n",
      "                                                                 C18[0][0]                        \n",
      "                                                                 C19[0][0]                        \n",
      "                                                                 C20[0][0]                        \n",
      "                                                                 C21[0][0]                        \n",
      "                                                                 C22[0][0]                        \n",
      "                                                                 C23[0][0]                        \n",
      "                                                                 C24[0][0]                        \n",
      "                                                                 C25[0][0]                        \n",
      "                                                                 C26[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_dense_layer_2 (Embedd [(None, 1, 8), (None 0           I1[0][0]                         \n",
      "                                                                 I2[0][0]                         \n",
      "                                                                 I3[0][0]                         \n",
      "                                                                 I4[0][0]                         \n",
      "                                                                 I5[0][0]                         \n",
      "                                                                 I6[0][0]                         \n",
      "                                                                 I7[0][0]                         \n",
      "                                                                 I8[0][0]                         \n",
      "                                                                 I9[0][0]                         \n",
      "                                                                 I10[0][0]                        \n",
      "                                                                 I11[0][0]                        \n",
      "                                                                 I12[0][0]                        \n",
      "                                                                 I13[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_4 (Concatenate)     (None, 39, 8)        0           embedding_sparse_layer_2[1][0]   \n",
      "                                                                 embedding_sparse_layer_2[1][1]   \n",
      "                                                                 embedding_sparse_layer_2[1][2]   \n",
      "                                                                 embedding_sparse_layer_2[1][3]   \n",
      "                                                                 embedding_sparse_layer_2[1][4]   \n",
      "                                                                 embedding_sparse_layer_2[1][5]   \n",
      "                                                                 embedding_sparse_layer_2[1][6]   \n",
      "                                                                 embedding_sparse_layer_2[1][7]   \n",
      "                                                                 embedding_sparse_layer_2[1][8]   \n",
      "                                                                 embedding_sparse_layer_2[1][9]   \n",
      "                                                                 embedding_sparse_layer_2[1][10]  \n",
      "                                                                 embedding_sparse_layer_2[1][11]  \n",
      "                                                                 embedding_sparse_layer_2[1][12]  \n",
      "                                                                 embedding_sparse_layer_2[1][13]  \n",
      "                                                                 embedding_sparse_layer_2[1][14]  \n",
      "                                                                 embedding_sparse_layer_2[1][15]  \n",
      "                                                                 embedding_sparse_layer_2[1][16]  \n",
      "                                                                 embedding_sparse_layer_2[1][17]  \n",
      "                                                                 embedding_sparse_layer_2[1][18]  \n",
      "                                                                 embedding_sparse_layer_2[1][19]  \n",
      "                                                                 embedding_sparse_layer_2[1][20]  \n",
      "                                                                 embedding_sparse_layer_2[1][21]  \n",
      "                                                                 embedding_sparse_layer_2[1][22]  \n",
      "                                                                 embedding_sparse_layer_2[1][23]  \n",
      "                                                                 embedding_sparse_layer_2[1][24]  \n",
      "                                                                 embedding_sparse_layer_2[1][25]  \n",
      "                                                                 embedding_dense_layer_2[1][0]    \n",
      "                                                                 embedding_dense_layer_2[1][1]    \n",
      "                                                                 embedding_dense_layer_2[1][2]    \n",
      "                                                                 embedding_dense_layer_2[1][3]    \n",
      "                                                                 embedding_dense_layer_2[1][4]    \n",
      "                                                                 embedding_dense_layer_2[1][5]    \n",
      "                                                                 embedding_dense_layer_2[1][6]    \n",
      "                                                                 embedding_dense_layer_2[1][7]    \n",
      "                                                                 embedding_dense_layer_2[1][8]    \n",
      "                                                                 embedding_dense_layer_2[1][9]    \n",
      "                                                                 embedding_dense_layer_2[1][10]   \n",
      "                                                                 embedding_dense_layer_2[1][11]   \n",
      "                                                                 embedding_dense_layer_2[1][12]   \n",
      "__________________________________________________________________________________________________\n",
      "attention_layer_4 (attention_la (None, 39, 18)       324         concatenate_4[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "attention_layer_5 (attention_la (None, 39, 18)       684         attention_layer_4[1][0]          \n",
      "__________________________________________________________________________________________________\n",
      "flatten_4 (Flatten)             (None, 702)          0           attention_layer_5[1][0]          \n",
      "__________________________________________________________________________________________________\n",
      "dense_58 (Dense)                (None, 1)            703         flatten_4[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 7,087,391\n",
      "Trainable params: 7,087,391\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "AutoInt_ctr.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
